{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import tensorflow.contrib as tc\n",
    "import gym\n",
    "from random import sample\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    }
   ],
   "source": [
    "gamma=0.99 \n",
    "tau=0.001 \n",
    "normalize_returns=False \n",
    "normalize_observations=True\n",
    "batch_size=128 \n",
    "observation_range=(0., 1000.) \n",
    "action_range=(-0.05, 0.05) \n",
    "return_range=(-np.inf, np.inf),\n",
    "critic_l2_reg=0. \n",
    "actor_lr=3e-4 #1e-5\n",
    "critic_lr=3e-5 #3e-5\n",
    "clip_norm=None \n",
    "reward_scale=1.\n",
    "\n",
    "env = gym.make(\"Pendulum-v0\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.]\n",
      "[-2.]\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "Tensor(\"state_encode/input_1:0\", shape=(?, ?, 3), dtype=float32)\n",
      "Model: \"model_1\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, None, 1)]    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            [(None, None, 3)]    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, None, 4)      0           input_2[0][0]                    \n",
      "                                                                 input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "gru (GRU)                       [(None, 64), (None,  13248       concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 256)          16640       gru[0][1]                        \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 1)            257         dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_policy/mul (TensorF [(None, 1)]          0           dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_policy/truediv (Ten [(None, 1)]          0           tf_op_layer_policy/mul[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "tf_op_layer_policy/add (TensorF [(None, 1)]          0           tf_op_layer_policy/truediv[0][0] \n",
      "==================================================================================================\n",
      "Total params: 30,145\n",
      "Trainable params: 30,145\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#state place holder\n",
    "obs_dim = env.observation_space.shape[0]\n",
    "action_dim = env.action_space.shape[0]\n",
    "action_high = env.action_space.high\n",
    "action_low = env.action_space.low\n",
    "print(action_high)\n",
    "print(action_low)\n",
    "\n",
    "#RNN network for state\n",
    "with tf.compat.v1.variable_scope(\"action_state_encode\"):\n",
    "    obs_input = tf.keras.layers.Input(shape = (None,obs_dim))\n",
    "    action_input = tf.keras.layers.Input(shape = (None,action_dim))\n",
    "    #obs_h = tf.keras.layers.Dense(4,activation = 'relu')(obs_input)\n",
    "    action_rnn_layer,action_state_h = tf.keras.layers.GRU(64,return_state=True)(tf.keras.layers.Concatenate()([action_input,obs_input]))\n",
    "    print(obs_input)\n",
    "    \n",
    "#RNN network for state\n",
    "with tf.compat.v1.variable_scope(\"value_state_encode\"):\n",
    "    #obs_h = tf.keras.layers.Dense(4,activation = 'relu')(obs_input)\n",
    "    value_rnn_layer,value_state_h = tf.keras.layers.GRU(64,return_state=True)(tf.keras.layers.Concatenate()([action_input,obs_input]))\n",
    "    print(obs_input)\n",
    "\n",
    "#policy network\n",
    "with tf.compat.v1.variable_scope(\"policy\"):\n",
    "    #output action\n",
    "    action_output_h = tf.keras.layers.Dense(256,activation = 'relu')(action_state_h)\n",
    "    action_output = tf.keras.layers.Dense(action_dim,activation = 'tanh')(action_output_h)*(action_high-action_low)/2+(action_low+action_high)/2\n",
    "    \n",
    "with tf.compat.v1.variable_scope(\"target_policy\"):\n",
    "    #output action\n",
    "    target_action_output_h = tf.keras.layers.Dense(256,activation = 'relu')(action_state_h)\n",
    "    target_action_output = tf.keras.layers.Dense(action_dim,activation = 'tanh')(action_output_h)*(action_high-action_low)/2+(action_low+action_high)/2\n",
    "\n",
    "    \n",
    "#output value function\n",
    "with tf.compat.v1.variable_scope(\"value_function\"):\n",
    "    value_h = tf.keras.layers.Dense(4,activation = 'relu')(value_state_h)\n",
    "    new_action_input = tf.keras.layers.Input(shape = (action_dim))\n",
    "    value_input = tf.keras.layers.Concatenate()([new_action_input,value_h])\n",
    "    value_h2 = tf.keras.layers.Dense(256,activation = 'relu')(value_input)\n",
    "    Q_value_output = tf.keras.layers.Dense(1,activation = 'linear')(value_h2)\n",
    "    \n",
    "with tf.compat.v1.variable_scope(\"target_value_function\"):\n",
    "    target_value_h = tf.keras.layers.Dense(4,activation = 'relu')(value_state_h)\n",
    "    target_new_action_input = tf.keras.layers.Input(shape = (action_dim))\n",
    "    target_value_input = tf.keras.layers.Concatenate()([target_new_action_input, target_value_h])\n",
    "    target_value_h2 = tf.keras.layers.Dense(256,activation = 'relu')(target_value_input)\n",
    "    target_Q_value_output = tf.keras.layers.Dense(1,activation = 'linear')(target_value_h2)\n",
    "\n",
    "#actor, critic model\n",
    "model = tf.keras.Model([obs_input,action_input,new_action_input],[action_output,Q_value_output])\n",
    "actor_model = tf.keras.Model([obs_input,action_input],action_output)\n",
    "critic_model = tf.keras.Model([obs_input,action_input,new_action_input],Q_value_output)\n",
    "target_actor_model = tf.keras.Model([obs_input,action_input],target_action_output)\n",
    "target_critic_model = tf.keras.Model([obs_input,action_input,target_new_action_input],target_Q_value_output)\n",
    "actor_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/training/moving_averages.py:433: Variable.initialized_value (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use Variable.read_value. Variables in 2.X are initialized automatically both in eager and graph (inside tf.defun) contexts.\n"
     ]
    }
   ],
   "source": [
    "#computation graph\n",
    "#policy gradient ascent\n",
    "Q_grad = tf.gradients(critic_model.output,new_action_input)\n",
    "Q_action_grad = tf.compat.v1.placeholder(tf.float32, shape=(None,action_dim))\n",
    "actor_model_weights = actor_model.trainable_weights\n",
    "action_grad = tf.gradients(actor_model.output,actor_model_weights,-Q_action_grad)\n",
    "act_grad_ascent = tf.train.AdamOptimizer(actor_lr).apply_gradients(zip(action_grad,actor_model_weights))\n",
    "\n",
    "#critic mse\n",
    "adam  = tf.keras.optimizers.Adam(critic_lr)\n",
    "critic_model.compile(loss=\"mse\", optimizer=adam)\n",
    "\n",
    "#target networks\n",
    "#moving average\n",
    "ema = tf.train.ExponentialMovingAverage(decay=0.99)\n",
    "policy_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"policy\")\n",
    "value_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"value_function\")\n",
    "target_policy_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"target_policy\")\n",
    "target_value_weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,\"target_value_function\")\n",
    "target_policy_upd = ema.apply(policy_weights)\n",
    "target_value_upd = ema.apply(value_weights)\n",
    "target_policy_assign = tf.group([tf.assign(target_policy_weights[i], ema.average(policy_weights[i])) for i in range(len(policy_weights))])\n",
    "target_value_assign = tf.group([tf.assign(target_value_weights[i], ema.average(value_weights[i])) for i in range(len(value_weights))])\n",
    "#ema.average(critic_model_weights)\n",
    "\n",
    "\n",
    "#initialization\n",
    "sess = tf.Session()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "195 {'loss': [38.923171043395996]}\n",
      "epoch:  0\n",
      "rewards:  -1392.8579666364599\n",
      "195 {'loss': [39.594122886657715]}\n",
      "195 {'loss': [34.948400020599365]}\n",
      "195 {'loss': [20.272123336791992]}\n",
      "195 {'loss': [16.232497930526733]}\n",
      "195 {'loss': [11.321239471435547]}\n",
      "195 {'loss': [9.603323698043823]}\n",
      "195 {'loss': [12.277170062065125]}\n",
      "195 {'loss': [12.313477039337158]}\n",
      "195 {'loss': [9.537736654281616]}\n",
      "195 {'loss': [9.20673954486847]}\n",
      "epoch:  10\n",
      "rewards:  -1335.2337319279177\n",
      "195 {'loss': [6.969679832458496]}\n",
      "195 {'loss': [6.7913830280303955]}\n",
      "195 {'loss': [4.385062158107758]}\n",
      "195 {'loss': [4.255519151687622]}\n",
      "195 {'loss': [3.6625298857688904]}\n",
      "195 {'loss': [2.5010078251361847]}\n",
      "195 {'loss': [1.3967384994029999]}\n",
      "195 {'loss': [1.7011220753192902]}\n",
      "195 {'loss': [1.4789334833621979]}\n",
      "195 {'loss': [1.5104963183403015]}\n",
      "epoch:  20\n",
      "rewards:  -1406.34884227453\n",
      "195 {'loss': [1.030065581202507]}\n",
      "195 {'loss': [1.3937216252088547]}\n",
      "195 {'loss': [1.4787666648626328]}\n",
      "195 {'loss': [0.9364462792873383]}\n",
      "195 {'loss': [0.8745352625846863]}\n",
      "195 {'loss': [0.7061376422643661]}\n",
      "195 {'loss': [0.7161357551813126]}\n",
      "195 {'loss': [0.8455470651388168]}\n",
      "195 {'loss': [0.7959341332316399]}\n",
      "195 {'loss': [0.6491503939032555]}\n",
      "epoch:  30\n",
      "rewards:  -1498.1122708839312\n",
      "195 {'loss': [0.799079179763794]}\n",
      "195 {'loss': [0.8552712798118591]}\n",
      "195 {'loss': [0.49786177277565]}\n",
      "195 {'loss': [0.5665231347084045]}\n",
      "195 {'loss': [0.48656483367085457]}\n",
      "195 {'loss': [0.3813594728708267]}\n",
      "195 {'loss': [0.47279035300016403]}\n",
      "195 {'loss': [0.5100259482860565]}\n",
      "195 {'loss': [0.34112391620874405]}\n",
      "195 {'loss': [0.3355037793517113]}\n",
      "epoch:  40\n",
      "rewards:  -1609.916836646869\n",
      "195 {'loss': [0.3096017688512802]}\n",
      "195 {'loss': [0.47558002918958664]}\n",
      "195 {'loss': [0.4823763072490692]}\n",
      "195 {'loss': [0.3578382730484009]}\n",
      "195 {'loss': [0.3499425835907459]}\n",
      "195 {'loss': [0.6361644864082336]}\n",
      "195 {'loss': [0.27380772680044174]}\n",
      "195 {'loss': [0.31749991327524185]}\n",
      "195 {'loss': [0.2600397430360317]}\n",
      "195 {'loss': [0.27744971215724945]}\n",
      "epoch:  50\n",
      "rewards:  -1492.817250402301\n",
      "195 {'loss': [0.3139103651046753]}\n",
      "195 {'loss': [0.2810414247214794]}\n",
      "195 {'loss': [0.27143896371126175]}\n",
      "195 {'loss': [0.22475874423980713]}\n",
      "195 {'loss': [0.29048213362693787]}\n",
      "195 {'loss': [0.26681455969810486]}\n",
      "195 {'loss': [0.286527905613184]}\n",
      "195 {'loss': [0.3733140639960766]}\n",
      "195 {'loss': [0.19918405637145042]}\n",
      "195 {'loss': [0.2640416920185089]}\n",
      "epoch:  60\n",
      "rewards:  -1459.9933024569514\n",
      "195 {'loss': [0.2349870353937149]}\n",
      "195 {'loss': [0.3348957486450672]}\n",
      "195 {'loss': [0.16779651306569576]}\n",
      "195 {'loss': [0.2127125933766365]}\n",
      "195 {'loss': [0.16635675728321075]}\n",
      "195 {'loss': [0.15602971240878105]}\n",
      "195 {'loss': [0.21546628698706627]}\n",
      "195 {'loss': [0.17540032416582108]}\n",
      "195 {'loss': [0.13666372001171112]}\n",
      "195 {'loss': [0.22383597493171692]}\n",
      "epoch:  70\n",
      "rewards:  -1407.4584957157426\n",
      "195 {'loss': [0.15188412740826607]}\n",
      "195 {'loss': [0.3518754206597805]}\n",
      "195 {'loss': [0.17018046602606773]}\n",
      "195 {'loss': [0.15036198496818542]}\n",
      "195 {'loss': [0.14638864807784557]}\n",
      "195 {'loss': [0.14575081318616867]}\n",
      "195 {'loss': [0.15938366018235683]}\n",
      "195 {'loss': [0.13579569198191166]}\n",
      "195 {'loss': [0.15543395280838013]}\n",
      "195 {'loss': [0.2007502131164074]}\n",
      "epoch:  80\n",
      "rewards:  -1457.631895544309\n",
      "195 {'loss': [0.14123773574829102]}\n",
      "195 {'loss': [0.10515803843736649]}\n",
      "195 {'loss': [0.1239880733191967]}\n",
      "195 {'loss': [0.26122443564236164]}\n",
      "195 {'loss': [0.10458955727517605]}\n",
      "195 {'loss': [0.29398365691304207]}\n",
      "195 {'loss': [0.2843354009091854]}\n",
      "195 {'loss': [0.1335699912160635]}\n",
      "195 {'loss': [0.10053950548171997]}\n",
      "195 {'loss': [0.17934999614953995]}\n",
      "epoch:  90\n",
      "rewards:  -1418.5004082842931\n",
      "195 {'loss': [0.12360063008964062]}\n",
      "195 {'loss': [0.11625607497990131]}\n",
      "195 {'loss': [0.1613941639661789]}\n",
      "195 {'loss': [0.1030573919415474]}\n",
      "195 {'loss': [0.22473261132836342]}\n",
      "195 {'loss': [0.09271840564906597]}\n",
      "195 {'loss': [0.11421056091785431]}\n",
      "195 {'loss': [0.1518833450973034]}\n",
      "195 {'loss': [0.08151474688202143]}\n",
      "195 {'loss': [0.11651481688022614]}\n",
      "epoch:  100\n",
      "rewards:  -1490.6054653670344\n",
      "195 {'loss': [0.09113476239144802]}\n",
      "195 {'loss': [0.09008915536105633]}\n",
      "195 {'loss': [0.08264543022960424]}\n",
      "195 {'loss': [0.0901576429605484]}\n",
      "195 {'loss': [0.08950302097946405]}\n",
      "195 {'loss': [0.09745693020522594]}\n",
      "195 {'loss': [0.12199030071496964]}\n",
      "195 {'loss': [0.08314454648643732]}\n",
      "195 {'loss': [0.14632184896618128]}\n",
      "195 {'loss': [0.08514285832643509]}\n",
      "epoch:  110\n",
      "rewards:  -1463.0754065726837\n",
      "195 {'loss': [0.14570101164281368]}\n",
      "195 {'loss': [0.12914678268134594]}\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-84da1c0322d4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     91\u001b[0m                 \u001b[0mobs_input\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0mobs_seq_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0maction_input\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mact_seq_data\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                 \u001b[0mQ_action_grad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mQ_action_grads\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m             })\n\u001b[1;32m     95\u001b[0m             \u001b[0;31m#train critic\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1339\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1427\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1431\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "EPOCH = 2000\n",
    "max_step = 200\n",
    "max_seq = 2\n",
    "batch_size = 128\n",
    "epsilon = 0.1\n",
    "\n",
    "replay_buffer = []\n",
    "max_size = 10000\n",
    "rewards_history = []\n",
    "\n",
    "for epoch in range(EPOCH):\n",
    "    #print(\"epoch: \",epoch)\n",
    "    obs_seq_list = []\n",
    "    act_seq_list = []\n",
    "    obs = env.reset()\n",
    "    obs = obs.reshape((obs_dim,))\n",
    "    obs_seq_list.append(obs)\n",
    "    act_seq_list.append(np.zeros(action_dim))\n",
    "    for t in range(max_step):\n",
    "        #interaction\n",
    "        obs_seq = np.asarray(obs_seq_list)\n",
    "        act_seq = np.asarray(act_seq_list)\n",
    "        obs_seq = obs_seq.reshape((1,-1,obs_dim))\n",
    "        act_seq = act_seq.reshape((1,-1,action_dim))\n",
    "        \n",
    "        #pad input\n",
    "        pad_width = max_seq-np.size(obs_seq,1)\n",
    "        next_pad_width = max_seq-np.size(obs_seq,1)+1\n",
    "        obs_seq = np.pad(obs_seq,((0,0),(pad_width,0),(0,0)))\n",
    "        act_seq = np.pad(act_seq,((0,0),(pad_width,0),(0,0)))\n",
    "        \n",
    "        #choose action\n",
    "        r = np.random.rand(1)\n",
    "        if r>epsilon:\n",
    "            action = sess.run(action_output,feed_dict={obs_input:obs_seq,action_input:act_seq})\n",
    "        else:\n",
    "            action = env.action_space.sample()\n",
    "        \n",
    "        action = action.reshape((action_dim,))\n",
    "        next_obs,reward,done,_ = env.step(action)\n",
    "        next_obs = next_obs.reshape((obs_dim,))\n",
    "        \n",
    "        obs_seq_list.append(next_obs)\n",
    "        act_seq_list.append(action)\n",
    "        #keep the length same\n",
    "        if len(obs_seq_list)>max_seq:\n",
    "            obs_seq_list.pop(0)\n",
    "            act_seq_list.pop(0)\n",
    "        next_obs_seq = np.asarray(obs_seq_list)\n",
    "        next_act_seq = np.asarray(act_seq_list)\n",
    "        next_obs_seq = next_obs_seq.reshape((1,-1,obs_dim))\n",
    "        next_act_seq = next_act_seq.reshape((1,-1,action_dim))\n",
    "        \n",
    "        #pad data\n",
    "        pad_width = max_seq-np.size(obs_seq,1)\n",
    "        next_pad_width = max_seq-np.size(next_obs_seq,1)\n",
    "        obs_seq = np.pad(obs_seq,((0,0),(pad_width,0),(0,0)))\n",
    "        act_seq = np.pad(act_seq,((0,0),(pad_width,0),(0,0)))\n",
    "        next_obs_seq = np.pad(next_obs_seq,((0,0),(next_pad_width,0),(0,0)))\n",
    "        next_act_seq = np.pad(next_act_seq,((0,0),(next_pad_width,0),(0,0)))\n",
    "        \n",
    "        #record\n",
    "        replay_buffer.append([obs_seq,act_seq,action,reward,next_obs_seq,next_act_seq,done])\n",
    "        if len(replay_buffer)>max_size:\n",
    "            replay_buffer.pop(0)\n",
    "        \n",
    "        #move to the next\n",
    "        obs = next_obs\n",
    "        \n",
    "        \n",
    "        #train\n",
    "        #get data from the buffer\n",
    "        if len(replay_buffer)>batch_size and t%5==0 and t!=0:\n",
    "            train_datas = sample(replay_buffer,batch_size)\n",
    "            array = np.asarray(train_datas)\n",
    "            obs_seq_data = np.stack(array[:,0]).reshape((batch_size,max_seq,-1))\n",
    "            act_seq_data = np.stack(array[:,1]).reshape((batch_size,max_seq,-1))\n",
    "            action_data = np.stack(array[:,2]).reshape((batch_size,-1))\n",
    "            reward_data = np.stack(array[:,3]).reshape((batch_size,-1))\n",
    "            next_obs_data = np.stack(array[:,4]).reshape((batch_size,max_seq,-1))\n",
    "            next_act_data = np.stack(array[:,5]).reshape((batch_size,max_seq,-1))\n",
    "            done_data = np.stack(array[:,6]).reshape((batch_size,-1))\n",
    "\n",
    "            #train actor\n",
    "            Q_action_grads = sess.run(Q_grad, feed_dict={\n",
    "                obs_input:  obs_seq_data,\n",
    "                action_input: act_seq_data,\n",
    "                new_action_input: action_data,\n",
    "            })[0]\n",
    "            sess.run(act_grad_ascent, feed_dict={\n",
    "                obs_input:  obs_seq_data,\n",
    "                action_input: act_seq_data,\n",
    "                Q_action_grad:Q_action_grads\n",
    "            })\n",
    "            #train critic\n",
    "            target_actions = sess.run(target_action_output, feed_dict = {\n",
    "                obs_input: next_obs_data,\n",
    "                action_input: next_act_data,\n",
    "            })\n",
    "            target_Qs = sess.run(target_Q_value_output, feed_dict = {\n",
    "                obs_input: next_obs_data,\n",
    "                action_input: next_act_data,\n",
    "                target_new_action_input: target_actions\n",
    "            })\n",
    "            Q_y = reward_data + gamma*target_Qs*(1-done)\n",
    "            evaluation = critic_model.fit([obs_seq_data,act_seq_data,action_data], Q_y, verbose=0)\n",
    "            if t==max_step-5:\n",
    "                print(t,evaluation.history)\n",
    "            #update target\n",
    "            sess.run(target_policy_upd)\n",
    "            sess.run(target_value_upd)\n",
    "            sess.run(target_policy_assign)\n",
    "            sess.run(target_value_assign)\n",
    "        \n",
    "        if done:\n",
    "            break\n",
    "                \n",
    "    #evaluate\n",
    "    if epoch%10==0:\n",
    "        obs_seq_list = []\n",
    "        act_seq_list = []\n",
    "        ave_rewards = 0\n",
    "        for times in range(5):\n",
    "            rewards = 0\n",
    "            obs = env.reset()          \n",
    "            for t in range(500):\n",
    "                #interaction\n",
    "                obs_seq = np.asarray(obs_seq_list)\n",
    "                act_seq = np.asarray(act_seq_list)\n",
    "                obs_seq = obs_seq.reshape((1,-1,obs_dim))\n",
    "                act_seq = act_seq.reshape((1,-1,action_dim))\n",
    "                #pad input\n",
    "                pad_width = max_seq-np.size(obs_seq,1)\n",
    "                obs_seq = np.pad(obs_seq,((0,0),(pad_width,0),(0,0)))\n",
    "                act_seq = np.pad(act_seq,((0,0),(pad_width,0),(0,0)))\n",
    "                #choose action\n",
    "                action = sess.run(action_output,feed_dict={obs_input:obs_seq,action_input:act_seq})\n",
    "                action = action.reshape((action_dim,))\n",
    "                next_obs,reward,done,_ = env.step(action)\n",
    "                obs = next_obs.reshape((obs_dim,))\n",
    "                obs_seq_list.append(obs)   \n",
    "                act_seq_list.append(action)\n",
    "                if len(obs_seq_list)>max_seq:\n",
    "                    obs_seq_list.pop(0)\n",
    "                    act_seq_list.pop(0)\n",
    "\n",
    "                rewards = rewards+reward\n",
    "                if done:\n",
    "                    break\n",
    "            ave_rewards = ave_rewards+rewards\n",
    "            \n",
    "        ave_rewards = ave_rewards/5\n",
    "        rewards_history.append(ave_rewards)\n",
    "        print(\"epoch: \",epoch)\n",
    "        print(\"rewards: \", ave_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = [0,1,2,3]\n",
    "a.append(4)\n",
    "print(a)\n",
    "a.pop(0)\n",
    "print(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
